(conc) ubuntu@ip-172-31-0-52:~/serve$ pytest test/pytest/test_handler.py -k test_huggingface_bert_batch_inference  2>&1 | tee regression.txt
============================= test session starts ==============================
platform linux -- Python 3.9.7, pytest-6.2.5, py-1.10.0, pluggy-0.13.1
rootdir: /home/ubuntu/serve
plugins: mock-3.6.1, cov-3.0.0
collected 9 items / 8 deselected / 1 selected

test/pytest/test_handler.py .                                            [100%]

================= 1 passed, 8 deselected in 177.50s (0:02:57) ==================

2021-10-07 00:29:49,365 [INFO ] W-9000-BERTSeqClassification_1.0-stdout MODEL_LOG - INFERENCE LIST
2021-10-07 00:29:49,365 [INFO ] W-9000-BERTSeqClassification_1.0-stdout MODEL_LOG - ['Not Accepted', 'Not Accepted']
2021-10-07 00:29:49,365 [INFO ] W-9000-BERTSeqClassification_1.0-stdout MODEL_LOG - LENGTH OF INFERENCE
2021-10-07 00:29:49,365 [INFO ] W-9000-BERTSeqClassification_1.0-stdout MODEL_LOG - 2

java.util.ConcurrentModificationException
        at java.base/java.util.LinkedHashMap$LinkedHashIterator.nextNode(LinkedHashMap.java:719)
        at java.base/java.util.LinkedHashMap$LinkedKeyIterator.next(LinkedHashMap.java:741)
        at org.pytorch.serve.wlm.BatchAggregator.sendResponse(BatchAggregator.java:81)
        at org.pytorch.serve.wlm.WorkerThread.run(WorkerThread.java:194)
        at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base/java.lang.Thread.run(Thread.java:829)

    async def run_async_inference():
        await run_inference_using_url_with_data(TF_INFERENCE_API + '/v1/models/BERTSeqClassification:predict', pfiles=files, ptimeout=1000)


    loop = asyncio.get_event_loop()
    tasks = []
    for i in range(batch_size - 1):
        task = loop.create_task(run_async_inference())
        tasks.append(task)
    
    loop.run_until_complete(asyncio.gather(*tasks))

   def invoke():
        res = run_inference_using_url_with_data(TF_INFERENCE_API + '/v1/models/BERTSeqClassification:predict', pfiles=files, ptimeout=1000)                                                                        
        return res

    pool = multiprocessing.Pool(batch_size)
    results = pool.map(invoke, batch_size*[])
    pool.close()
    pool.join()
    print(results)